import re
import datetime
import wsgiref.handlers
import logging
log = logging.getLogger("spider")

from google.appengine.ext import webapp
from google.appengine.api import urlfetch
from google.appengine.ext import db

import dataparser
import statsdb

class Spider(webapp.RequestHandler):
  def get(self):
    url = self.request.get('url')
    result = urlfetch.fetch(url)
    
    self.response.headers['Content-Type'] = 'text/plain'
    if 200 != result.status_code:
      self.response.out.write(result.status_code)
    else:
      parser = dataparser.DataParser()
      charset = "UTF-8"
      m = re.match(".* charset=(.*)", result.headers["Content-Type"], re.IGNORECASE)
      if None != m:
        charset = m.groups()[0]
      content = self.checkTrash(result.content.decode(charset))
      if len(self.request.get('date')) > 0:
        parser.date = (datetime.datetime.strptime(self.request.get('date'), "%Y%m%d%H%M%S"), "%Y")
      if len(self.request.get('dfmt')) > 0:
        parser.date = (parser.date[0], self.request.get('dfmt'))
      if len(self.request.get('unit')) > 0:
        parser.unit = self.request.get('unit')
      op = "view"
      if len(self.request.get('op')) > 0:
        op = self.request.get('op')
      parser.feed(content)
      parser.close()
      if "view" == op:
        self.response.out.write("date,dfmt,value,unit,tags,source\n")
        for rec in parser.records:
          self.response.out.write(rec[2][0].strftime("%Y-%m-%d") + "," + rec[2][1] + "," + str(rec[0])
                                  + ","+rec[1]+"," + " ".join(rec[3]) + "," + url + "\n")
      elif "write" == op:
        records = {}
        for rec in parser.records:
          if True == statsdb.GetRecord(rec[2][0], rec[3])[0] \
          or True == records.has_key(statsdb.GenRecordKeyName(rec[2][0], rec[3])):
            log.warn("The Record[%s, %s] already exists, ignored",
                     rec[2][0].strftime("%Y-%m-%d"), "|".join(rec[3]))
            self.response.out.write("Ignored->" +rec[2][0].strftime(rec[2][1]) + ": "
                                    + str(rec[0]) +"("+rec[1]+")" +" [" + "|".join(rec[3]) +"]\n")
            continue
          record = statsdb.Record(key_name = statsdb.GenRecordKeyName(rec[2][0], rec[3]),
                                  value = rec[0],
                                  unit = rec[1],
                                  date = rec[2][0],
                                  dfmt = rec[2][1],
                                  tags = rec[3],
                                  source = url)
          records[statsdb.GenRecordKeyName(rec[2][0], rec[3])] = record
          log.debug("Written->%s:%.1f(%s)[%s]", record.date.strftime(record.dfmt),
                    record.value, record.unit, "|".join(record.tags))
          self.response.out.write("Written->" +rec[2][0].strftime(rec[2][1]) + ": "
                                  + str(rec[0]) +"("+rec[1]+")" +" [" + "|".join(rec[3]) +"]\n")
        batch = 50
        if len(self.request.get('batch')) > 0:
          batch = int(self.request.get('batch'))
        statsdb.DBBatch(db.put, records.values(), batch)
        statsdb.UpdateEntity(records, True)
        self.response.out.write(str(len(records)) + " records written.\n")
  
  def repl2(self, m):
    s = m.group()
    return s.replace(s, s[0:2]+" "+s[2])

  def repl1(self, m):
    return re.sub('(\w[\'"]\w)', self.repl2, m.group())

  def checkTrash(self, content):
    return re.sub('(<[^<^>]*>)', self.repl1, content)

def main():
  application = webapp.WSGIApplication([('/spider', Spider)], 
                                       debug=True)
  wsgiref.handlers.CGIHandler().run(application)

if __name__ == "__main__":
  main()
